# -*- coding: UTF-8 -*-
from pyspark import SparkConf, SparkContext
from pyspark.sql import SQLContext
from pyspark.sql.types import *
import datetime
import sys
import pymongo
import re

waitR = """
.houseJrtt { background:#FCFCFC; border:1px solid #DEDEDE; border-bottom:2px solid #E5E5E5;  width:658px; clear:both; margin:30px auto 12px; position:relative;}
.houseJrtt .c1 { width:658px; clear:both; margin:0 auto; border-bottom:1px solid #DEDEDE;}
    .houseJrtt .c1 .c1L { position:absolute; background:url(http://img1.cache.netease.com/house/2015/5/14/201505141323501ff32.gif) no-repeat 0 0; width:66px; height:77px; top:-5px; left:7px;}
.houseJrtt .c1 .c1L a { width:66px; height:77px; display:block; text-decoration:none;}
.houseJrtt .c1 .c1M { width:420px; float:left; margin-left:85px; display:inline;}
.houseJrtt .c1 .c1M h3 {font-family:"\5FAE\8F6F\96C5\9ED1","\9ED1\4F53","\5B8B\4F53"; font-weight:bold; font-size:19px; color:#252525; line-height:23px; padding:13px 0 5px; text-align:left;}
#endText .houseJrtt .c1 .c1M h3 a ,#endText .houseJrtt .c1 .c1M h3 a:visited { color:#252525; text-decoration:none;}
#endText .houseJrtt .c1 .c1M h3 a:hover { color:#bc2931;}
.houseJrtt .c1 .c1M h4 { text-align:left; line-height:21px; color:#888888; background:url(http://img1.cache.netease.com/house/img14/housearc1401/hbg04.gif) no-repeat 0 12px; padding:3px 0 2px 10px; font-size:12px;}
.houseJrtt .c1 .c1M .house_under_links { background:none; padding-left: 0px;}
#endText .houseJrtt .c1 .c1M h4 a ,#endText .houseJrtt .c1 .c1M h4 a:visited { color:#888888; text-decoration:none;}
#endText .houseJrtt .c1 .c1M h4 a:hover { color:#bc2931;}
.houseJrtt .c1 .c1R { width:65px; text-align:center; float:right; margin-right:4px; display:inline; font-size:12px; padding-top:4px;}
.houseJrtt .c1 .c1R .k1 { clear:both; width:100%; text-align:center;}
.houseJrtt .c1 .c1R .k2 { font-size:12px; width:100%; display:block; text-align:center; clear:both; margin:0 auto; color:#969696; line-height:20px;}
#endText .houseJrtt .c1 .c1R .k2 a ,#endText .houseJrtt .c1 .c1R .k2 a:visited { color:#969696; text-decoration:none;}
#endText .houseJrtt .c1 .c1R .k2 a:hover { color:#bc2931;}
.w9 .houseJrtt { background:#FCFCFC; border-top:1px solid #DEDEDE; border-bottom:2px solid #E5E5E5;  width:580px; clear:both; margin:30px auto 12px; position:relative;}
    .w9 .houseJrtt .c1 { width:580px; clear:both; margin:0 auto; border-bottom:1px solid #DEDEDE;}
    .w9 .houseJrtt .c1 .c1L { position:absolute; background:url(http://img1.cache.netease.com/house/2015/5/14/201505141323501ff32.gif) no-repeat 0 0; width:66px; height:77px; top:-5px; left:7px;}
.w9 .houseJrtt .c1 .c1L a { width:66px; height:77px; display:block; text-decoration:none;}
.w9 .houseJrtt .c1 .c1M { width:420px; float:left; margin-left:85px; display:inline;}
.w9 .houseJrtt .c1 .c1M h3 {font-family:"\5FAE\8F6F\96C5\9ED1","\9ED1\4F53","\5B8B\4F53"; font-weight:bold; font-size:19px; color:#252525; line-height:23px; padding:13px 0 5px; text-align:left;}
.w9 #endText .houseJrtt .c1 .c1M h3 a ,#endText .houseJrtt .c1 .c1M h3 a:visited { color:#252525; text-decoration:none;}
    .w9 #endText .houseJrtt .c1 .c1M h3 a:hover { color:#bc2931;}
    .w9 .houseJrtt .c1 .c1M h4 { text-align:left; line-height:21px; color:#888888; background:url(http://img1.cache.netease.com/house/img14/housearc1401/hbg04.gif) no-repeat 0 12px; padding:3px 0 2px 10px; font-size:12px;}
.w9 .houseJrtt .c1 .c1M .house_under_links { background:none; padding-left: 0px;}
.w9 #endText .houseJrtt .c1 .c1M h4 a ,#endText .houseJrtt .c1 .c1M h4 a:visited { color:#888888; text-decoration:none;}
    .w9 #endText .houseJrtt .c1 .c1M h4 a:hover { color:#bc2931;}
    .w9 .houseJrtt .c1 .c1R { width:65px; text-align:center; float:right; margin-right:4px; display:inline; font-size:12px; padding-top:4px;}
.w9 .houseJrtt .c1 .c1R .k1 { clear:both; width:100%; text-align:center;}
.w9 .houseJrtt .c1 .c1R .k2 { font-size:12px; width:100%; display:block; text-align:center; clear:both; margin:0 auto; color:#969696; line-height:20px;}
.w9 #endText .houseJrtt .c1 .c1R .k2 a ,#endText .houseJrtt .c1 .c1R .k2 a:visited { color:#969696; text-decoration:none;}
    .w9 #endText .houseJrtt .c1 .c1R .k2 a:hover { color:#bc2931;}





北京二手房降价超200万元 交易量恐进一步萎缩

房产首页 |
买房推荐 |
楼盘查询 |
看房团 |
直通车


房产北京站




本文来源：新京报
责任编辑：汪瑞杰_NO7469

"""

db = pymongo.MongoClient(host="127.0.0.1", port=27017).openCourse.college_html
dbInser = pymongo.MongoClient(host="127.0.0.1", port=27017).openCourse.college_article_number

# college_types = ["CMU",
#                  "MIT",
#                  "JHSPH",
#                  "DUKE",
#                  "Yale",
#                  "HMS",
#                  "PCE",
#                  "USU",
#                  "TUFTS",
#                  "UMB",
#                  "BC",
#                  "UMICH",
#                  "OX",
#                  "AC",
#                  "Nottingham",
#                  "OpenLearn",
#                  "NewSu",
#                  "Tudelft",
#                  "VTC"]
college_types = ['EDX']
for college in college_types:
    htmls = db.find({'college':{'$regex':'EDX'}})
    MIT = {"college":'EDX'}
    MIT['fiveK'] = 0
    MIT['threeK'] = 0
    MIT['oneK'] = 0
    MIT['fiveH'] = 0
    MIT['fiveHL'] = 0
    MIT['allNum'] = 0
    for html in htmls:
        htmlLength = html['articleLength']
        if htmlLength >= 5000:
            MIT['fiveK'] += 1
        if htmlLength >= 3000 and htmlLength < 5000:
            MIT['threeK'] += 1
        if htmlLength >= 1000 and htmlLength < 3000:
            MIT['oneK'] += 1
        if htmlLength >= 500 and htmlLength < 1000:
            MIT['fiveH'] += 1
        if htmlLength < 500:
            MIT['fiveHL'] += 1
        MIT['allNum'] += 1
    #将数据写入
    dbInser.insert(MIT)
#
# def subStrInfo(str):
#     r = u'[’!"#$%&\'()*+,-./:;<=>?@，：\\n\\r\\t。\ \\?★、…【】《》？“”‘’！[\\]^_`{|}~]+'
#     try:
#         filterStr = re.sub(r, " ", str)
#     except Exception as e:
#         filterStr = str
#     return filterStr
#
#
# for html in htmls:
#     tempStr = subStrInfo(html['title'])
#     if tempStr == html['title']:
#         pass
#     else:
#         # strLength = len(tempStr)
#         print(db.update({"_id":html['_id']},{"$set":{"title":tempStr}}))

#5000字以上 3000字以上 1000字以上 500字以上 500字以下

